package com.bocloud.cmp.service.app.resource;

import java.io.File;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

import com.bocloud.cmp.entity.app.resource.Application;
import com.bocloud.cmp.entity.app.resource.ApplicationLayoutInfo;
import com.bocloud.cmp.entity.app.resource.ApplicationOpenshiftClusterInfo;
import com.bocloud.cmp.entity.app.resource.ApplicationSoftwareInfo;
import com.bocloud.cmp.entity.app.resource.DeployHistory;
import com.bocloud.cmp.entity.app.resource.Layout;
import com.bocloud.cmp.entity.app.resource.OpenshiftCluster;
import com.bocloud.cmp.entity.app.resource.RegistryImageInfo;
import com.bocloud.cmp.entity.app.resource.Software;
import com.bocloud.cmp.enums.app.ApplicationEnum;
import com.bocloud.cmp.enums.app.DeployHistoryEnum;
import com.bocloud.cmp.interfaces.app.resource.ApplicationService;
import com.bocloud.cmp.service.model.OperateResult;
import com.bocloud.cmp.service.utils.OpenshiftUtil;
import com.bocloud.cmp.service.utils.PodResourceInfo;
import com.bocloud.common.model.BsmResult;
import com.bocloud.common.model.GridBean;
import com.bocloud.common.model.Param;
import com.bocloud.common.model.RequestUser;
import com.bocloud.common.model.SimpleBean;
import com.bocloud.common.utils.Common;
import com.bocloud.common.utils.GridHelper;
import com.bocloud.common.utils.ListTool;
import com.bocloud.soc.entity.Authority;
import com.bocloud.soc.entity.Role;
import com.bocloud.soc.entity.User;
import com.google.common.collect.Maps;

import io.fabric8.kubernetes.api.model.Container;
import io.fabric8.kubernetes.api.model.HasMetadata;
import io.fabric8.kubernetes.api.model.Pod;
import io.fabric8.kubernetes.api.model.Quantity;
import io.fabric8.kubernetes.api.model.ReplicationController;
import io.fabric8.kubernetes.api.model.ResourceRequirements;
import io.fabric8.kubernetes.api.model.ServiceBuilder;
import io.fabric8.kubernetes.api.model.extensions.HorizontalPodAutoscaler;

/**
 * @author zjm
 * @date 2017年3月17日
 */
@Service("applicationService")
public class ApplicationServiceImpl extends CommonServiceImpl implements ApplicationService {

	private static Logger logger = LoggerFactory.getLogger(ApplicationServiceImpl.class);

	/**
	 * 江苏银行特殊属性，滚动后休眠然后重启service
	 */
	@Value("${rolling.sleep}")
	private Integer rollingSleep;

	@Autowired
	private StatisticAppServiceImpl statisticAppServiceImpl;

	@Override
	public BsmResult list(int page, int rows, List<Param> params, Map<String, String> sorter, Boolean simple,
			RequestUser requestUser) {
		List<Application> list;
		List<SimpleBean> beans;
		GridBean gridBean;
		BsmResult bsmResult = new BsmResult();
		try {
			String deptId = queryUserDeptId(requestUser.getId());
			if (null == sorter) {
				sorter = Maps.newHashMap();
			}
			sorter.put("gmtCreate", Common.ONE);
			int total = 0;
			total = applicationDao.count(params, deptId);
			if (simple) {
				beans = applicationDao.list(params, sorter, deptId);
				gridBean = new GridBean(1, 1, total, beans);
			} else {
				list = applicationDao.list(page, rows, params, sorter, deptId);
				// 为所有应用设置实例数
				for (Application application : list) {
					Map<String, String> appLabels;
					if (null == (appLabels = getAppLabels(application.getId(), bsmResult))) {
						continue;
					}
					bsmResult = getClusterProXyUrl(application.getId());
					if (bsmResult.isSuccess() && null != bsmResult.getData()) {
						OpenshiftUtil.TwoTuple<List<Pod>, String> twoTuple = oseUtil
								.getPods(bsmResult.getData().toString(), appLabels);
						List<Pod> pods = twoTuple.first;
						if (!ListTool.isEmpty(pods)) {
							application.setReplicas(pods.size());
						}
					}
				}
				gridBean = GridHelper.getBean(page, rows, total, list);
			}
			bsmResult.setSuccess(true);
			bsmResult.setData(gridBean);
			bsmResult.setMessage("应用列表查询成功！");
		} catch (Exception e) {
			logger.error("list application failure:", e);
			bsmResult.setSuccess(false);
			bsmResult.setMessage("应用列表查询失败！");
		}
		saveLog("application/operation", "应用列表查询", bsmResult.getMessage(), requestUser.getId());
		return bsmResult;
	}

	@Override
	public BsmResult listByLayout(int page, int rows, List<Param> params, Map<String, String> sorter, Boolean simple,
			RequestUser requestUser) {
		List<Application> list;
		GridBean gridBean;
		BsmResult bsmResult = new BsmResult();
		try {
			String deptId = queryUserDeptId(requestUser.getId());
			if (null == sorter) {
				sorter = Maps.newHashMap();
			}
			sorter.put("gmtCreate", Common.ONE);
			int total = 0;
			total = applicationDao.countByLayout(params, deptId);
			list = applicationDao.listByLayout(page, rows, params, sorter, deptId);
			// 为所有应用设置实例数
			for (Application application : list) {
				Map<String, String> appLabels = getAppLabels(application.getId(), new BsmResult());
				bsmResult = getClusterProXyUrl(application.getId());
				if (bsmResult.isSuccess() && null != bsmResult.getData()) {
					bsmResult = oseUtil.getRcsReplicas(bsmResult.getData().toString(), appLabels);
					if (bsmResult.isSuccess() && null != bsmResult.getData()) {
						application.setReplicas((Integer) bsmResult.getData());
					}
				}
			}
			gridBean = GridHelper.getBean(page, rows, total, list);
			bsmResult.setSuccess(true);
			bsmResult.setData(gridBean);
			bsmResult.setMessage("编排应用列表查询成功！");
		} catch (Exception e) {
			logger.error("list application failure:", e);
			bsmResult.setSuccess(false);
			bsmResult.setMessage("编排应用列表查询失败！");
		}
		saveLog("application/operation", "编排应用列表查询", bsmResult.getMessage(), requestUser.getId());
		return bsmResult;
	}

	@Override
	public BsmResult create(RequestUser requestUser, Application application) {
		BsmResult bsmResult = new BsmResult();
		try {
			if (null != (applicationDao.queryByName(application.getName()))) {
				logger.warn("应用创建失败！应用名称[" + application.getName() + "]已经存在！");
				bsmResult.setMessage("应用创建失败！应用名称已经存在！");
				return bsmResult;
			}
			User user = null;
			if (null == (user = queryUser(requestUser.getId(), bsmResult))) {
				return bsmResult;
			}
			application.setStatus(ApplicationEnum.Status.NOT_DEPLOY.toString());
			application.setDeleted(false);
			application.setCreaterId(requestUser.getId());
			application.setGmtCreate(new Date());
			application.setCreaterId(requestUser.getId());
			application.setMenderId(requestUser.getId());
			application.setOwnerId(requestUser.getId());
			application.setTenantId(requestUser.getTenantId());
			application.setDeptId(user.getDepartId());
			applicationDao.save(application);
			bsmResult.setSuccess(true);
			bsmResult.setMessage("应用创建成功!");
		} catch (Exception e) {
			logger.error("create application [" + application.getName() + "] failure:", e);
			bsmResult.setSuccess(false);
			bsmResult.setMessage("应用创建失败！");
		} finally {
			saveLog("application/operation", "应用创建", bsmResult.getMessage(), requestUser.getId());
		}
		return bsmResult;
	}

	@Override
	public BsmResult detail(Long id, RequestUser user) {
		BsmResult bsmResult = new BsmResult();
		Application application = null;
		if (null == (application = queryApplication(id, bsmResult))) {
			saveLog("application/operation", "应用详情", bsmResult.getMessage(), user.getId());
			return bsmResult;
		}
		bsmResult.setSuccess(true);
		bsmResult.setData(application);
		bsmResult.setMessage("应用查询详情成功！");
		saveLog("application/operation", "应用详情", bsmResult.getMessage(), user.getId());
		return bsmResult;
	}

	@Override
	public BsmResult remove(List<Long> ids, Long userId) {
		BsmResult bsmResult = new BsmResult();
		int successCount = 0;
		for (Long id : ids) {
			Application application = null;
			try {
				if (null == (application = queryApplication(id, bsmResult))) {
					continue;
				}
				if (application.getStatus().equals(ApplicationEnum.Status.EXECUTE_DEPLOY.toString())) {
					bsmResult.setMessage("应用[" + application.getName() + "]正在部署不允许删除！");
					logger.error(bsmResult.getMessage());
					return bsmResult;
				}
				Map<String, String> appLabels = getAppLabels(id, bsmResult);
				if (null == appLabels) {
					continue;
				}
				bsmResult = getClusterProXyUrl(id);
				if (bsmResult.isSuccess() && null != bsmResult.getData()) {
					String proxyUrl = bsmResult.getData().toString();
					// 获取所有rc
					if (null != (bsmResult = oseUtil.getRcs(proxyUrl, appLabels)).getData()) {
						@SuppressWarnings("all")
						List<ReplicationController> rcs = (List<ReplicationController>) bsmResult.getData();
						// 循环删除rc和部署历史
						for (ReplicationController rc : rcs) {
							if (!(bsmResult = deleteRc(id, rc.getMetadata().getNamespace(), rc.getMetadata().getName(),
									userId)).isSuccess()) {
								continue;
							}
						}
					}
					// TODO 暂不支持Deployment资源的管理，请不要删除
					// // 删除deployment
					// if (!(bsmResult = deleteDeployments(proxyUrl,
					// appLabels)).isSuccess()) {
					// return bsmResult;
					// }
					// 删除service
					if (!(bsmResult = deleteServices(proxyUrl, appLabels)).isSuccess()) {
						return bsmResult;
					}
					// 删除pod
					if (!(bsmResult = deletePods(proxyUrl, appLabels)).isSuccess()) {
						return bsmResult;
					}
					// 删除hpa
					if (!(bsmResult = deleteHpas(proxyUrl, appLabels)).isSuccess()) {
						return bsmResult;
					}
				}
				if (!removeAppInfos(id, userId)) {
					bsmResult.setSuccess(false);
					bsmResult.setMessage("应用[" + application.getName() + "]删除失败！");
					return bsmResult;
				}
				successCount++;
				logger.info("remove application[" + application.getName() + "]success！");
			} catch (Exception e) {
				logger.error("remove application[" + application.getName() + "] error:", e);
				bsmResult.setSuccess(false);
				bsmResult.setMessage("应用[" + application.getName() + "]删除失败！");
				return bsmResult;
			} finally {
				saveLog("application/operation", "删除应用", bsmResult.getMessage(), userId);
			}
		}
		bsmResult.setSuccess(false);
		if (successCount == ids.size()) {
			logger.info("remove application success");
			bsmResult.setSuccess(true);
			bsmResult.setMessage("应用删除成功！");
		} else if (successCount == 0) {
			bsmResult.setMessage("应用删除成功！");
		} else {
			bsmResult.setMessage("应用删除部分成功！请检查日志！");
		}
		return bsmResult;
	}

	@Override
	public BsmResult modify(Application application, Long userId) {
		BsmResult bsmResult = new BsmResult();
		Application updateApplication = null;
		if (null == (updateApplication = queryApplication(application.getId(), bsmResult))) {
			saveLog("application/operation", "应用信息修改", bsmResult.getMessage(), userId);
			return bsmResult;
		}
		updateApplication.setMenderId(userId);
		updateApplication.setGmtModify(new Date());
		updateApplication.setName(application.getName());
		updateApplication.setRemark(application.getRemark());
		try {
			if (this.applicationDao.update(updateApplication)) {
				bsmResult.setSuccess(true);
				bsmResult.setMessage("应用修改成功！");
			} else {
				bsmResult.setSuccess(false);
				bsmResult.setMessage("应用修改失败！");
			}
		} catch (Exception e) {
			logger.error("modify application[" + application.getName() + "] fail:", e);
			bsmResult.setSuccess(false);
			bsmResult.setMessage("应用修改失败！");
		} finally {
			saveLog("application/operation", "应用信息修改", bsmResult.getMessage(), userId);
		}
		return bsmResult;
	}

	@Override
	public BsmResult layoutDeploy(Long appId, Long layoutId, Long clusterId, RequestUser user) {
		BsmResult bsmResult = new BsmResult();
		Application application;
		if (null == (application = queryApplication(appId, bsmResult))) {
			saveLog("application/operation", "编排部署", bsmResult.getMessage(), user.getId());
			return bsmResult;
		}
		Layout layout;
		if (null == (layout = queryLayout(layoutId, bsmResult))) {
			saveLog("application/operation", "编排部署", bsmResult.getMessage(), user.getId());
			return bsmResult;
		}
		OpenshiftCluster openshiftCluster;
		if (null == (openshiftCluster = queryOpenshiftCluster(clusterId, bsmResult))) {
			saveLog("application/operation", "编排部署", bsmResult.getMessage(), user.getId());
			return bsmResult;
		}
		try {
			// 启动一个线程从MQ上获取数据
			new Thread(new Runnable() {
				@Override
				public void run() {
					BsmResult bsmResult = new BsmResult();
					try {
						// 修改应用状态为部署中
						application.setStatus(ApplicationEnum.Status.EXECUTE_DEPLOY.toString());
						modifyApplication(application, bsmResult);
						bsmResult = oseUtil.layoutDeploy(openshiftCluster.getProxyUrl(), application.getName(),
								layout.getFilePath() + File.separatorChar + layout.getFileName());

						// 如果编排部署失败判断是否有data（所有pod），如果有需要将所有pod的事件保存到log里面
						if (!bsmResult.isSuccess()) {
							if (null != bsmResult.getData()) {
								String logResult = "";
								@SuppressWarnings("unchecked")
								List<Pod> pods = (List<Pod>) bsmResult.getData();
								for (Pod pod : pods) {
									bsmResult = statisticAppServiceImpl.event(pod.getMetadata().getNamespace(),
											pod.getKind(), pod.getMetadata().getName(), clusterId);
									if (bsmResult.isSuccess() && null != bsmResult.getData()) {
										logResult += bsmResult.getData().toString();
									}
								}
								logger.error("编排部署失败：\n" + logResult);
								resourceEventPublisher.send(new OperateResult(false, "编排部署失败！相关pod事件请查看[系统日志]",
										"application/layoutDeploy", user.getId()));
								saveLog("application/operation", "编排部署", logResult, user.getId());
								return;
							} else {
								logger.error(bsmResult.getMessage());
								resourceEventPublisher.send(new OperateResult(false, bsmResult.getMessage(),
										"application/layoutDeploy", user.getId()));
								saveLog("application/operation", "编排部署", bsmResult.getMessage(), user.getId());
								return;
							}
						}
						@SuppressWarnings("all")
						List<HasMetadata> deploySourceList = (List<HasMetadata>) bsmResult.getData();
						if (ListTool.isEmpty(deploySourceList)) {
							logger.warn(bsmResult.getMessage());
							resourceEventPublisher.send(new OperateResult(false, bsmResult.getMessage(),
									"application/layoutDeploy", user.getId()));
							saveLog("application/operation", "编排部署", bsmResult.getMessage(), user.getId());
							return;
						}
						// 如果编排部署执行成功，则添加应用中间表和修改应用信息
						if (bsmResult.isSuccess()) {
							if ((bsmResult = insertApplicationMiddleTable(application, layoutId, clusterId))
									.isSuccess()) {
								// 循环所有文件资源，如果是rc或者deployment则添加部署历史和镜像数据
								saveDeployHistroyAndSoftwareByResource(deploySourceList, appId, bsmResult, user);
								if (bsmResult.isSuccess()) {
									bsmResult.setMessage("编排部署成功！");
									logger.info(bsmResult.getMessage());
								} else {
									logger.error(bsmResult.getMessage());
									bsmResult.setMessage("编排部署失败！" + bsmResult.getMessage());
									oseUtil.deleteLayoutDeploy(openshiftCluster.getProxyUrl(),
											layout.getFilePath() + File.separatorChar + layout.getFileName());
								}
							} else {
								logger.error(bsmResult.getMessage());
								bsmResult.setMessage("编排部署失败！" + bsmResult.getMessage());
								oseUtil.deleteLayoutDeploy(openshiftCluster.getProxyUrl(),
										layout.getFilePath() + File.separatorChar + layout.getFileName());
							}
						} else {
							logger.error(bsmResult.getMessage());
							bsmResult.setMessage("编排部署失败！" + bsmResult.getMessage());
							oseUtil.deleteLayoutDeploy(openshiftCluster.getProxyUrl(),
									layout.getFilePath() + File.separatorChar + layout.getFileName());
						}
						saveLog("application/operation", "编排部署", bsmResult.getMessage(), user.getId());
						resourceEventPublisher.send(new OperateResult(bsmResult.isSuccess(), bsmResult.getMessage(),
								"application/layoutDeploy", bsmResult.getData(), user.getId()));
						return;
					} catch (Exception e) {
						logger.error(bsmResult.getMessage());
						resourceEventPublisher.send(new OperateResult(false, bsmResult.getMessage(),
								"application/layoutDeploy", user.getId()));
						saveLog("application/operation", "编排部署", bsmResult.getMessage(), user.getId());
						return;
					} finally {
						// 修改应用状态为已部署
						logger.info("执行完部署后，修改应用的部署状态为已部署");
						application.setStatus(ApplicationEnum.Status.DEPLOY.toString());
						modifyApplication(application, bsmResult);
					}
				}
			}).start();
			return new BsmResult(true, "编排文件部署任务已经下发，正在执行……");
		} catch (Exception e) {
			logger.error("layout deploy failure:", e);
			bsmResult.setSuccess(false);
			bsmResult.setMessage("编排文件部署失败!");
			saveLog("application/operation", "编排部署", bsmResult.getMessage(), user.getId());
			return bsmResult;
		}
	}

	@Override
	public BsmResult deleteLayoutDeploy(Long appId, Long layoutId, RequestUser user) {
		BsmResult bsmResult = new BsmResult();
		Layout layout;
		if (null == (layout = queryLayout(layoutId, bsmResult))) {
			saveLog("application/operation", "删除编排部署", bsmResult.getMessage(), user.getId());
			return bsmResult;
		}
		bsmResult = getClusterProXyUrl(appId);
		if (bsmResult.isSuccess() && null != bsmResult.getData()) {
			bsmResult = oseUtil.deleteLayoutDeploy(bsmResult.getData().toString(),
					layout.getFilePath() + File.separatorChar + layout.getFileName());
		}
		saveLog("application/operation", "删除编排部署", bsmResult.getMessage(), user.getId());
		return bsmResult;
	}

	@Override
	public BsmResult scaleRc(Long appId, String namespace, String name, int count, RequestUser user) {
		BsmResult bsmResult = getClusterProXyUrl(appId);
		if (!bsmResult.isSuccess() || null == bsmResult.getData()) {
			return bsmResult;
		}
		String proxyUrl = bsmResult.getData().toString();
		// 判断当前rc是否正在滚动
		if (null == (bsmResult = oseUtil.getRc(proxyUrl, namespace, name)).getData()) {
			return bsmResult;
		}
		ReplicationController currentRc = (ReplicationController) bsmResult.getData();
		Map<String, String> lables = currentRc.getMetadata().getLabels();
		Map<String, String> testLables = new HashMap<>();
		testLables.put(OpenshiftUtil.DIFF_LABEL_KEY, lables.get(OpenshiftUtil.DIFF_LABEL_KEY));
		testLables.put(OpenshiftUtil.UNIFIED_LABEL_KEY, lables.get(OpenshiftUtil.UNIFIED_LABEL_KEY));
		if (!(bsmResult = oseUtil.getRcs(proxyUrl, testLables)).isSuccess()) {
			bsmResult.setMessage("修改实例失败：" + bsmResult.getMessage());
			logger.error(bsmResult.getMessage());
			saveLog("application/operation", "修改实例", bsmResult.getMessage(), user.getId());
			return bsmResult;
		}
		@SuppressWarnings("unchecked")
		List<ReplicationController> rcs = (List<ReplicationController>) bsmResult.getData();
		if (rcs == null || rcs.size() > 1) {
			logger.warn("current rc rolling...");
			bsmResult.setSuccess(false);
			bsmResult.setMessage("当前正在滚动升级或回滚，不支持修改实例");
			saveLog("application/operation", "修改实例", bsmResult.getMessage(), user.getId());
			return bsmResult;
		}
		try {
			// 启动一个线程从MQ上获取数据
			new Thread(new Runnable() {
				@Override
				public void run() {
					BsmResult bsmResult = oseUtil.scaleRc(proxyUrl, namespace, name, count);
					if (bsmResult.isSuccess()) {
						logger.info(bsmResult.getMessage());
					} else {
						logger.error(bsmResult.getMessage());
					}
					saveLog("application/operation", "修改实例", bsmResult.getMessage(), user.getId());
					resourceEventPublisher.send(new OperateResult(bsmResult.isSuccess(), bsmResult.getMessage(),
							"application/scaleRc", bsmResult.getData(), user.getId()));
					return;
				}
			}).start();
			return new BsmResult(true, "修改实例任务已经下发，正在执行……");
		} catch (Exception e) {
			logger.error("application scale failure:", e);
			bsmResult.setSuccess(false);
			bsmResult.setMessage("修改实例失败：\n" + e);
			saveLog("application/operation", "修改实例", bsmResult.getMessage(), user.getId());
			return bsmResult;
		}
	}
	// // TODO 暂不支持Deployment资源的管理，请不要删除
	// @Override
	// public BsmResult getDeployments(Long appId) {
	// BsmResult bsmResult = new BsmResult();
	// Map<String, String> appLabels = getAppLabels(appId, bsmResult);
	// if (null == appLabels) {
	// return bsmResult;
	// }
	// bsmResult = getClusterProXyUrl(appId);
	// if (bsmResult.isSuccess() && null != bsmResult.getData()) {
	// bsmResult = oseUtil.getDeployments(bsmResult.getData().toString(),
	// appLabels);
	// }
	// return bsmResult;
	// }

	@Override
	public BsmResult getServices(Long appId) {
		BsmResult bsmResult = new BsmResult();
		Map<String, String> appLabels = getAppLabels(appId, bsmResult);
		if (null == appLabels) {
			return bsmResult;
		}
		bsmResult = getClusterProXyUrl(appId);
		if (bsmResult.isSuccess() && null != bsmResult.getData()) {
			bsmResult = oseUtil.getServices(bsmResult.getData().toString(), appLabels);
		}
		return bsmResult;
	}

	@Override
	public BsmResult getPodStatus(Long appId, String name, String namespace, RequestUser user) {
		BsmResult bsmResult = getClusterProXyUrl(appId);
		if (!bsmResult.isSuccess() || null == bsmResult.getData()) {
			saveLog("application/operation", "查看运行实例的状态", bsmResult.getMessage(), user.getId());
			return bsmResult;
		}
		OpenshiftUtil.TwoTuple<Pod, String> twoTuple = oseUtil.getPod(bsmResult.getData().toString(), namespace, name);
		if (twoTuple.first == null) {
			saveLog("application/operation", "查看运行实例的状态", bsmResult.getMessage(), user.getId());
			return new BsmResult(false, twoTuple.second);
		}
		Pod pod = twoTuple.first;
		bsmResult.setData(pod.getStatus().getPhase());
		if (pod.getStatus().getPhase().equals("Running") || pod.getStatus().getPhase().equals("Succeeded")) {
			// bsmResult.setStatus(ApplicationEnum.Status.RUN.toString());
			bsmResult.setMessage("应用实例运行成功！");
			saveLog("application/operation", "查看运行实例的状态", bsmResult.getMessage(), user.getId());
			return bsmResult;
		} else if (pod.getStatus().getPhase().equals("Failed") || pod.getStatus().getPhase().equals("Unknown")) {
			// bsmResult.setStatus(ApplicationEnum.Status.ABNORMAL.toString());
			bsmResult.setMessage("应用实例运行失败！");
			saveLog("application/operation", "查看运行实例的状态", bsmResult.getMessage(), user.getId());
			return bsmResult;
		}
		// bsmResult.setStatus(ApplicationEnum.Status.EXECUTE_DEPLOY.toString());
		bsmResult.setMessage("应用实例部署中！");
		saveLog("application/operation", "查看运行实例的状态", bsmResult.getMessage(), user.getId());
		return bsmResult;
	}

	@Override
	public BsmResult rolling(String namespace, String resourceName, Long appId, Long softwareId, RequestUser user) {
		String operation = softwareId == null ? "回滚" : "滚动升级";
		BsmResult bsmResult = new BsmResult();
		// 当前rc
		ReplicationController currentRc = null;
		DeployHistory deployHistory = new DeployHistory();
		// 需要滚动（回滚）到的镜像
		String imageName;
		bsmResult = getClusterProXyUrl(appId);
		if (!bsmResult.isSuccess() || null == bsmResult.getData()) {
			saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
			return bsmResult;
		}
		String proxyUrl = bsmResult.getData().toString();
		bsmResult = oseUtil.getRc(proxyUrl, namespace, resourceName);
		if (bsmResult.isSuccess()) {
			currentRc = (ReplicationController) bsmResult.getData();
		}
		if (!bsmResult.isSuccess()) {
			bsmResult.setMessage(operation + "失败！" + bsmResult.getMessage());
			saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
			return bsmResult;
		}
		List<Container> containers = currentRc.getSpec().getTemplate().getSpec().getContainers();
		if (null == containers || containers.size() != 1) {
			bsmResult.setSuccess(false);
			bsmResult.setMessage(operation + "失败！仅支持资源中有且仅有一个容器时才能进行" + operation);
			saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
			return bsmResult;
		}
		// 如果镜像id不为空，则按照镜像id获取升级的镜像滚动升级，否则按照资源名称查询部署历史查询上一个版本的镜像进行回滚µ
		Software oldSoftware;
		String oldSoftwareName = containers.get(0).getImage();
		String oldSoftwareVersion = "latest";
		String oldSoftwareNamespace = "";
		if (oldSoftwareName.contains("/")) {
			if (oldSoftwareName.substring(0, oldSoftwareName.indexOf("/")).contains(":5000")) {
				oldSoftwareNamespace = (oldSoftwareName.substring(oldSoftwareName.indexOf("/") + 1));
				if (oldSoftwareNamespace.contains("/")) {
					oldSoftwareNamespace = oldSoftwareNamespace.substring(0, oldSoftwareNamespace.indexOf("/"));
				} else {
					oldSoftwareNamespace = "";
				}
				oldSoftwareName = (oldSoftwareName.substring(oldSoftwareName.indexOf("/") + 1))
						.substring((oldSoftwareName.substring(oldSoftwareName.indexOf("/") + 1).indexOf("/") + 1));
			} else {
				oldSoftwareNamespace = oldSoftwareName.substring(0, oldSoftwareName.indexOf("/"));
				oldSoftwareName = oldSoftwareName.substring(oldSoftwareName.indexOf("/") + 1);
			}
		}
		if (oldSoftwareName.contains(":")) {
			oldSoftwareVersion = oldSoftwareName.substring(oldSoftwareName.lastIndexOf(":") + 1);
			oldSoftwareName = oldSoftwareName.substring(0, oldSoftwareName.lastIndexOf(":"));
		}
		try {
			oldSoftware = softwareDao.getSoftwareByNameWithVersionAndNamespace(oldSoftwareName, oldSoftwareVersion,
					oldSoftwareNamespace);
		} catch (Exception e) {
			bsmResult.setSuccess(false);
			bsmResult.setMessage(operation + "失败！获取镜像信息失败！");
			logger.error(bsmResult.getMessage(), e);
			saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
			return bsmResult;
		}
		if (null == oldSoftware) {
			bsmResult.setSuccess(false);
			bsmResult.setMessage(operation + "失败！获取镜像信息失败！");
			logger.error(bsmResult.getMessage());
			saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
			return bsmResult;
		}
		Software newSoftware;
		if (null != softwareId) {
			if (null == (newSoftware = querySoftware(softwareId, new BsmResult()))) {
				bsmResult.setSuccess(false);
				bsmResult.setMessage(operation + "失败！获取镜像信息失败！");
				saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
				return bsmResult;
			}
			RegistryImageInfo registryImageInfo = new RegistryImageInfo();
			if (null == (registryImageInfo = queryRegistryImageInfo(softwareId, bsmResult))) {
				saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
				return bsmResult;
			}
			if (StringUtils.isNotBlank(registryImageInfo.getNamespace())) {
				imageName = registryImageInfo.getNamespace() + "/" + newSoftware.getName() + ":"
						+ newSoftware.getVersion();
			} else {
				imageName = newSoftware.getName() + ":" + newSoftware.getVersion();
			}
			if (StringUtils.isBlank(imageName)) {
				bsmResult.setSuccess(false);
				bsmResult.setMessage(operation + "失败！获取镜像信息失败！");
				saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
				return bsmResult;
			}
		} else {
			String currentImage = currentRc.getSpec().getTemplate().getSpec().getContainers().get(0).getImage();
			DeployHistory deployHistories;
			try {
				deployHistories = deployHistoryDao.getByResourceNameAndImage(resourceName, currentImage);
			} catch (Exception e) {
				bsmResult.setSuccess(false);
				bsmResult.setMessage("当前资源没有部署成功历史无法回滚!");
				logger.error(bsmResult.getMessage(), e);
				saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
				return bsmResult;
			}
			if (null == deployHistories) {
				bsmResult.setSuccess(false);
				bsmResult.setMessage("当前资源没有部署成功历史无法回滚!");
				logger.error(bsmResult.getMessage());
				saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
				return bsmResult;
			}
			imageName = deployHistories.getPreviousVersion();
			if (StringUtils.isBlank(imageName)) {
				bsmResult.setSuccess(false);
				bsmResult.setMessage("当前资源没有部署成功历史无法回滚!");
				logger.error(bsmResult.getMessage());
				saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
				return bsmResult;
			}
			String newSoftwareName = imageName;
			String newSoftwareVersion = "latest";
			String newSoftwareNamespace = "";
			// 将镜像信息去除命名空间和ip地址等信息
			if (newSoftwareName.contains("/")) {
				if (newSoftwareName.substring(0, newSoftwareName.indexOf("/")).contains(":5000")) {
					newSoftwareNamespace = (newSoftwareName.substring(newSoftwareName.indexOf("/") + 1));
					if (newSoftwareNamespace.contains("/")) {
						newSoftwareNamespace = newSoftwareNamespace.substring(0, newSoftwareNamespace.indexOf("/"));
					} else {
						newSoftwareNamespace = "";
					}
					newSoftwareName = (newSoftwareName.substring(newSoftwareName.indexOf("/") + 1))
							.substring((newSoftwareName.substring(newSoftwareName.indexOf("/") + 1).indexOf("/") + 1));
				} else {
					newSoftwareNamespace = newSoftwareName.substring(0, newSoftwareName.indexOf("/"));
					newSoftwareName = newSoftwareName.substring(newSoftwareName.indexOf("/") + 1);
				}
			}
			if (newSoftwareName.contains(":")) {
				newSoftwareVersion = newSoftwareName.substring(newSoftwareName.lastIndexOf(":") + 1);
				newSoftwareName = newSoftwareName.substring(0, newSoftwareName.lastIndexOf(":"));
			}
			try {
				newSoftware = softwareDao.getSoftwareByNameWithVersionAndNamespace(newSoftwareName, newSoftwareVersion,
						newSoftwareNamespace);
			} catch (Exception e) {
				bsmResult.setSuccess(false);
				bsmResult.setMessage(operation + "失败！获取镜像信息失败！");
				logger.error(bsmResult.getMessage(), e);
				saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
				return bsmResult;
			}
		}
		try {
			// 启动一个线程从MQ上获取数据
			new Thread(new Runnable() {
				@Override
				public void run() {
					BsmResult bsmResult = oseUtil.rollingRc(proxyUrl, namespace, resourceName, imageName);
					if (bsmResult.isSuccess()) {
						// 如果应用镜像表中没有该记录则添加，否则count加1
						try {
							ApplicationSoftwareInfo newApplicationSoftwareInfo = queryApplicationSoftwareByAppIdAndSoftwareId(
									appId, newSoftware.getId(), bsmResult);
							if (null == newApplicationSoftwareInfo) {
								ApplicationSoftwareInfo applicationSoftwareInfo = new ApplicationSoftwareInfo();
								applicationSoftwareInfo.setSoftwareId(newSoftware.getId());
								applicationSoftwareInfo.setApplicationId(appId);
								applicationSoftwareInfo.setUseCount(1);
								applicationSoftwareInfo.setSoftwareName(imageName);
								saveApplicationSoftwareInfo(applicationSoftwareInfo, bsmResult);
								deleteSurplusImage(appId);
							} else {
								newApplicationSoftwareInfo.setUseCount(newApplicationSoftwareInfo.getUseCount() + 1);
								applicationSoftwareInfoDao.updateApplicationSoftwareInfo(newApplicationSoftwareInfo);
							}
						} catch (Exception e) {
							bsmResult.setSuccess(false);
							bsmResult.setMessage(operation + "失败！添加应用依赖镜像失败！");
							logger.error(bsmResult.getMessage(), e);
							saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
							resourceEventPublisher
									.send(new OperateResult(false, bsmResult.getMessage(), "application/rolling"));
							return;
						}

						// // rc升级成功后，删除老的依赖镜像，添加新的依赖镜像
						// ApplicationSoftwareInfo oldApplicationSoftwareInfo =
						// queryApplicationSoftwareByAppIdAndSoftwareId(
						// appId, oldSoftware.getId(), bsmResult);
						// if (null != oldApplicationSoftwareInfo) {
						// try {
						// // 如果老的依赖数大于1则count数减1，否则删除
						// if (oldApplicationSoftwareInfo.getUseCount() > 1) {
						// oldApplicationSoftwareInfo
						// .setUseCount(oldApplicationSoftwareInfo.getUseCount()
						// - 1);
						// applicationSoftwareInfoDao
						// .updateApplicationSoftwareInfo(oldApplicationSoftwareInfo);
						// } else {
						// applicationSoftwareInfoDao
						// .deleteApplicationSoftwareInfo(oldApplicationSoftwareInfo);
						// }
						// } catch (Exception e) {
						// bsmResult.setSuccess(false);
						// bsmResult.setMessage(operation + "失败！删除应用依赖镜像失败！");
						// logger.error(bsmResult.getMessage(), e);
						// saveLog("application/operation", operation,
						// bsmResult.getMessage(), user.getId());
						// resourceEventPublisher
						// .send(new OperateResult(false,
						// bsmResult.getMessage(), "application/rolling"));
						// return;
						// }
						// }
						deployHistory.setPreviousVersion(containers.get(0).getImage());
						deployHistory.setStatus(DeployHistoryEnum.Status.SUCCESS.toString());
					}
					if (!bsmResult.isSuccess()) {
						deployHistory.setStatus(DeployHistoryEnum.Status.FAIL.toString());
					}
					deployHistory.setAppId(appId);
					deployHistory.setName(resourceName);
					deployHistory.setResourceType("ReplicationController");
					deployHistory.setResourceName(resourceName);
					deployHistory.setCurrentVersion(imageName);
					deployHistory.setDeleted(false);
					deployHistory.setGmtCreate(new Date());
					deployHistory.setGmtModify(new Date());
					deployHistory.setCreaterId(user.getId());
					deployHistory.setMenderId(user.getId());
					deployHistory.setOwnerId(user.getId());
					deployHistory.setTenantId(user.getTenantId());
					if (!saveDeployHistroy(deployHistory, new BsmResult())) {
						if (bsmResult.isSuccess()) {
							bsmResult.setMessage(operation + "成功！添加部署历史失败！");
						} else {
							bsmResult.setMessage(operation + "失败！" + bsmResult.getMessage() + "添加部署历史失败！");
						}
						bsmResult.setSuccess(false);
						logger.error(bsmResult.getMessage());
						saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
						resourceEventPublisher
								.send(new OperateResult(false, bsmResult.getMessage(), "application/rolling"));
						return;
					}
					if (bsmResult.isSuccess()) {
						bsmResult.setMessage(operation + "成功！");
						logger.info(bsmResult.getMessage());
					} else {
						bsmResult.setMessage(operation + "失败！" + bsmResult.getMessage());
						logger.error(bsmResult.getMessage());
					}
					saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
					resourceEventPublisher.send(new OperateResult(bsmResult.isSuccess(), bsmResult.getMessage(),
							"application/rolling", bsmResult.getData(), user.getId()));
					return;
				}
			}).start();
			return new BsmResult(true, operation + "任务已经下发，正在执行……");
		} catch (Exception e) {
			logger.error("application rolling failure:", e);
			bsmResult.setSuccess(false);
			bsmResult.setMessage(operation + "失败!");
			saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
			return bsmResult;
		}
	}

	@Override
	public BsmResult autoScale(Long appId, String namespace, String resourceName, String requestValue, int minReplicas,
			int maxReplicas, int cpuTargetUtilization, RequestUser user) {
		try {
			// 启动一个线程从MQ上获取数据
			new Thread(new Runnable() {
				@Override
				public void run() {
					BsmResult bsmResult = new BsmResult();
					Application application = null;
					if (null == (application = queryApplication(appId, bsmResult))) {
						logger.warn(bsmResult.getMessage());
						resourceEventPublisher.send(new OperateResult(bsmResult.isSuccess(), bsmResult.getMessage(),
								"application/autoScale", user.getId()));
						saveLog("application/operation", "弹性伸缩", bsmResult.getMessage(), user.getId());
						return;
					}
					bsmResult = getClusterProXyUrl(appId);
					if (!bsmResult.isSuccess() || null == bsmResult.getData()) {
						logger.warn(bsmResult.getMessage());
						resourceEventPublisher.send(new OperateResult(bsmResult.isSuccess(), bsmResult.getMessage(),
								"application/autoScale", user.getId()));
						saveLog("application/operation", "弹性伸缩", bsmResult.getMessage(), user.getId());
						return;
					}
					String proxyUrl = bsmResult.getData().toString();
					// 获取rc判断request是否有值
					if (null == (bsmResult = oseUtil.getRc(proxyUrl, namespace, resourceName)).getData()) {
						logger.warn(bsmResult.getMessage());
						resourceEventPublisher.send(new OperateResult(bsmResult.isSuccess(), bsmResult.getMessage(),
								"application/autoScale", user.getId()));
						saveLog("application/operation", "弹性伸缩", bsmResult.getMessage(), user.getId());
						return;
					}
					ReplicationController rc = (ReplicationController) bsmResult.getData();
					if (null == rc.getSpec().getTemplate().getSpec().getContainers().get(0).getResources().getRequests()
							.get("cpu")) {
						logger.warn(bsmResult.getMessage());
						resourceEventPublisher.send(new OperateResult(false, "该资源没设置cpu的请求值，请先更改资源限制！",
								"application/autoScale", user.getId()));
						saveLog("application/operation", "弹性伸缩", bsmResult.getMessage(), user.getId());
						return;
					}
					// 如果这个资源有hpa控制器，则先删除hpa控制器
					Map<String, String> appLabels = getAppLabels(appId, bsmResult);
					List<HorizontalPodAutoscaler> hpas = oseUtil.getHPAs(proxyUrl, appLabels).first;
					if (null != hpas && hpas.size() > 0) {
						oseUtil.deleteHPAs(proxyUrl, hpas);
					}
					// 添加hpa控制器
					String hpaName = application.getName() + "-"
							+ RandomStringUtils.randomAlphanumeric(7).toLowerCase();
					bsmResult = oseUtil.createHPA(proxyUrl, appLabels, hpaName, rc.getKind(), namespace, resourceName,
							minReplicas, maxReplicas, cpuTargetUtilization);
					if (!bsmResult.isSuccess()) {
						bsmResult.setMessage("设置弹性伸缩失败！" + bsmResult.getMessage());
						logger.error(bsmResult.getMessage());
					} else {
						bsmResult.setMessage("设置弹性伸缩成功！");
						logger.info(bsmResult.getMessage());
					}
					resourceEventPublisher.send(new OperateResult(bsmResult.isSuccess(), bsmResult.getMessage(),
							"application/autoScale", bsmResult.getData(), user.getId()));
					saveLog("application/operation", "弹性伸缩", bsmResult.getMessage(), user.getId());
					return;
				}
			}).start();
			return new BsmResult(true, "应用弹性伸缩任务已经下发，正在执行……");
		} catch (Exception e) {
			logger.error("application auto scale failure:", e);
			BsmResult bsmResult = new BsmResult(false, "应用弹性伸缩失败!");
			saveLog("application/operation", "弹性伸缩", bsmResult.getMessage(), user.getId());
			return bsmResult;
		}
	}

	@Override
	public BsmResult editRcResource(Long appId, String namespace, String rcName, Map<String, String> limits,
			Map<String, String> requests, RequestUser user) {
		Map<String, Quantity> limitMap = new HashMap<>();
		for (Map.Entry<String, String> entry : limits.entrySet()) {
			limitMap.put(entry.getKey(), new Quantity(entry.getValue()));
		}
		Map<String, Quantity> requestMap = new HashMap<>();
		for (Map.Entry<String, String> entry : requests.entrySet()) {
			requestMap.put(entry.getKey(), new Quantity(entry.getValue()));
		}
		BsmResult bsmResult = getClusterProXyUrl(appId);
		if (bsmResult.isSuccess() && null != bsmResult.getData()) {
			bsmResult = oseUtil.editRcResource(bsmResult.getData().toString(), namespace, rcName, limitMap, requestMap);
		}
		saveLog("application/operation", "设置资源限制", bsmResult.getMessage(), user.getId());
		return bsmResult;
	}

	@Override
	public BsmResult getPodResource(Long appId, String namespace, String resourceName, String resourceType,
			RequestUser user) {
		if (!checkAuthority(user)) {
			saveLog("application/operation", "查看运行实例资源", "当前用户没有该操作的权限", user.getId());
			return new BsmResult(false, "当前用户没有该操作权限");
		}
		BsmResult bsmResult = getClusterProXyUrl(appId);
		if (!bsmResult.isSuccess() || null == bsmResult.getData()) {
			saveLog("application/operation", "查看运行实例资源", bsmResult.getMessage(), user.getId());
			return bsmResult;
		}
		PodResourceInfo podResourceInfo = new PodResourceInfo();
		ResourceRequirements respurce = null;
		String proxyUrl = bsmResult.getData().toString();
		if (resourceType.equals("ReplicationController")) {
			if (null == (bsmResult = oseUtil.getRc(proxyUrl, namespace, resourceName)).getData()) {
				saveLog("application/operation", "查看运行实例资源", bsmResult.getMessage(), user.getId());
				return bsmResult;
			}
			ReplicationController rc = (ReplicationController) bsmResult.getData();
			respurce = rc.getSpec().getTemplate().getSpec().getContainers().get(0).getResources();
			// TODO 暂不支持Deployment资源的管理，请不要删除
			// } else if (resourceType.equals("Deployment")) {
			// if (null == (bsmResult = oseUtil.getRc(proxyUrl, namespace,
			// resourceName)).getData()) {
			// return bsmResult;
			// }
			// Deployment deployment = (Deployment) bsmResult.getData();
			// respurce =
			// deployment.getSpec().getTemplate().getSpec().getContainers().get(0).getResources();
		} else {
			bsmResult.setSuccess(false);
			bsmResult.setMessage("获取组件pod资源信息失败！");
			saveLog("application/operation", "查看运行实例资源", bsmResult.getMessage(), user.getId());
			return bsmResult;
		}
		if (null == respurce) {
			bsmResult.setSuccess(false);
			bsmResult.setMessage("获取组件pod资源信息失败！");
			saveLog("application/operation", "查看运行实例资源", bsmResult.getMessage(), user.getId());
			return bsmResult;
		}
		if (null != respurce.getLimits()) {
			if (null != respurce.getLimits().get("cpu")) {
				podResourceInfo.setCpuLimit(respurce.getLimits().get("cpu").getAmount());
			}
			if (null != respurce.getLimits().get("memory")) {
				podResourceInfo.setMemoryLimit(respurce.getLimits().get("memory").getAmount());
			}
		}
		if (null != respurce.getRequests()) {
			if (null != respurce.getRequests().get("cpu")) {
				podResourceInfo.setCpuRequest(respurce.getRequests().get("cpu").getAmount());
			}
			if (null != respurce.getRequests().get("memory")) {
				podResourceInfo.setMemoryRequest(respurce.getRequests().get("memory").getAmount());
			}
		}
		bsmResult.setSuccess(true);
		bsmResult.setMessage("获取组件pod资源信息成功！");
		bsmResult.setData(podResourceInfo);
		saveLog("application/operation", "查看运行实例资源", bsmResult.getMessage(), user.getId());
		return bsmResult;
	}

	@Override
	public BsmResult getHPA(Long appId, String namespace, String resourceName, String resourceKind, RequestUser user) {
		if (!checkAuthority(user)) {
			saveLog("application/operation", "弹性伸缩列表展示", "当前用户没有该操作的权限", user.getId());
			return new BsmResult(false, "当前用户没有该操作权限");
		}
		BsmResult bsmResult = getClusterProXyUrl(appId);
		if (!bsmResult.isSuccess() || null == bsmResult.getData()) {
			saveLog("application/operation", "弹性伸缩列表展示", bsmResult.getMessage(), user.getId());
			return bsmResult;
		}
		String proxyUrl = bsmResult.getData().toString();
		Map<String, String> appLabels = getAppLabels(appId, bsmResult);
		if (null == appLabels) {
			saveLog("application/operation", "弹性伸缩列表展示", bsmResult.getMessage(), user.getId());
			return bsmResult;
		}
		Map<String, Integer> hpaMap = new HashMap<>();
		if (null != (bsmResult = oseUtil.getHPA(proxyUrl, namespace, appLabels, resourceKind, resourceName))
				.getData()) {
			HorizontalPodAutoscaler hpa = (HorizontalPodAutoscaler) bsmResult.getData();
			hpaMap.put("minReplicas", hpa.getSpec().getMinReplicas());
			hpaMap.put("maxReplicas", hpa.getSpec().getMaxReplicas());
			hpaMap.put("cpuTargetUtilization", hpa.getSpec().getCpuUtilization().getTargetPercentage());
		}
		bsmResult.setSuccess(true);
		bsmResult.setMessage("获取组件hpa资源信息成功！");
		bsmResult.setData(hpaMap);
		saveLog("application/operation", "弹性伸缩列表展示", bsmResult.getMessage(), user.getId());
		return bsmResult;
	}

	@Override
	public BsmResult deleteResource(Long appId, String namespace, String resourceName, String resourceType,
			RequestUser user) {
		BsmResult bsmResult = new BsmResult(false, "未找到[" + resourceType + "]的[" + resourceName + "]资源");
		if (resourceType.equals("ReplicationController")) {
			bsmResult = deleteRc(appId, namespace, resourceName, user.getId());
		} else if (resourceType.equals("Service")) {
			bsmResult = deleteService(appId, namespace, resourceName);
		} else if (resourceType.equals("Pod")) {
			bsmResult = deletePod(appId, namespace, resourceName);
		} else if (resourceType.equals("HorizontalPodAutoscaler")) {
			bsmResult = deleteHorizontalPodAutoscaler(appId, namespace, resourceName);
		}
		saveLog("application/operation", "删除资源", bsmResult.getMessage(), user.getId());
		return bsmResult;
	}

	public BsmResult deleteRc(Long appId, String namespace, String rcName, Long userId) {
		BsmResult bsmResult = getClusterProXyUrl(appId);
		if (!bsmResult.isSuccess() || null == bsmResult.getData()) {
			return bsmResult;
		}
		String proxyUrl = bsmResult.getData().toString();
		bsmResult = oseUtil.getRc(proxyUrl, namespace, rcName);
		if (!bsmResult.isSuccess()) {
			return bsmResult;
		}
		ReplicationController rc = (ReplicationController) bsmResult.getData();
		List<ReplicationController> rcs = new ArrayList<>();
		rcs.add(rc);
		bsmResult = oseUtil.deleteReplicationControllers(proxyUrl, rcs);
		// 如果rc删除成功，需要删除该rc的部署历史
		if (bsmResult.isSuccess()) {
			try {
				deployHistoryDao.deleteByResourceName(rc.getMetadata().getName(), userId);
			} catch (Exception e) {
				logger.error("删除[" + rc.getMetadata().getName() + "]资源的部署历史失败！", e);
				return new BsmResult(false, "删除[" + rc.getMetadata().getName() + "]资源的部署历史失败！");
			}
		}
		return bsmResult;
	}

	// TODO 暂不支持Deployment资源的管理，请不要删除
	// public BsmResult deleteDeployment(Long appId, String namespace, String
	// deploymentName, Long userId) {
	// BsmResult bsmResult = getClusterProXyUrl(appId);
	// if (!bsmResult.isSuccess() || null == bsmResult.getData()) {
	// return bsmResult;
	// }
	// String proxyUrl = bsmResult.getData().toString();
	// bsmResult = oseUtil.getDeployment(proxyUrl, namespace, deploymentName);
	// if (!bsmResult.isSuccess()) {
	// return bsmResult;
	// }
	// Deployment deployment = (Deployment) bsmResult.getData();
	// List<Deployment> deployments = new ArrayList<>();
	// deployments.add(deployment);
	// bsmResult = oseUtil.deleteDeployments(proxyUrl, deployments);
	// // 如果deployment删除成功，需要删除该deployment的部署历史
	// if (bsmResult.isSuccess()) {
	// try {
	// if
	// (!deployHistoryDao.deleteByResourceName(deployment.getMetadata().getName(),
	// userId)) {
	// return new BsmResult(false, "删除[" + deployment.getMetadata().getName() +
	// "]资源的部署历史失败！");
	// }
	// } catch (Exception e) {
	// logger.error("删除[" + deployment.getMetadata().getName() + "]资源的部署历史失败！",
	// e);
	// return new BsmResult(false, "删除[" + deployment.getMetadata().getName() +
	// "]资源的部署历史失败！");
	// }
	// }
	// return bsmResult;
	// }

	public BsmResult deleteService(Long appId, String namespace, String serviceName) {
		BsmResult bsmResult = getClusterProXyUrl(appId);
		if (!bsmResult.isSuccess() || null == bsmResult.getData()) {
			return bsmResult;
		}
		String proxyUrl = bsmResult.getData().toString();
		bsmResult = oseUtil.getService(proxyUrl, namespace, serviceName);
		if (!bsmResult.isSuccess()) {
			return bsmResult;
		}
		io.fabric8.kubernetes.api.model.Service service = (io.fabric8.kubernetes.api.model.Service) bsmResult.getData();
		List<io.fabric8.kubernetes.api.model.Service> services = new ArrayList<>();
		services.add(service);
		return oseUtil.deleteServices(proxyUrl, services);
	}

	public BsmResult deletePod(Long appId, String namespace, String podName) {
		BsmResult bsmResult = getClusterProXyUrl(appId);
		if (!bsmResult.isSuccess() || null == bsmResult.getData()) {
			return bsmResult;
		}
		String proxyUrl = bsmResult.getData().toString();
		OpenshiftUtil.TwoTuple<Pod, String> twoTuple = oseUtil.getPod(proxyUrl, namespace, podName);
		if (twoTuple.first == null) {
			return new BsmResult(false, twoTuple.second);
		}
		List<Pod> pods = new ArrayList<>();
		pods.add(twoTuple.first);
		return oseUtil.deletePods(proxyUrl, pods);
	}

	public BsmResult deleteHorizontalPodAutoscaler(Long appId, String namespace, String hpaName) {
		BsmResult bsmResult = getClusterProXyUrl(appId);
		if (!bsmResult.isSuccess() || null == bsmResult.getData()) {
			return bsmResult;
		}
		String proxyUrl = bsmResult.getData().toString();
		bsmResult = oseUtil.getHPA(proxyUrl, namespace, hpaName);
		if (!bsmResult.isSuccess()) {
			return bsmResult;
		}
		HorizontalPodAutoscaler hpa = (HorizontalPodAutoscaler) bsmResult.getData();
		List<HorizontalPodAutoscaler> hpas = new ArrayList<>();
		hpas.add(hpa);
		return oseUtil.deleteHPAs(proxyUrl, hpas);
	}

	/**
	 * 返回应用默认的label
	 * 
	 * @author zjm
	 * @date 2017年3月23日
	 *
	 * @param appId
	 * @return
	 */
	@Override
	public Map<String, String> getAppLabels(Long appId, BsmResult bsmResult) {
		Application application;
		if (null == (application = queryApplication(appId, bsmResult)) || StringUtils.isBlank(application.getName())) {
			logger.error("获取应用[" + appId + "]信息失败！");
			bsmResult.setSuccess(false);
			bsmResult.setMessage("获取应用信息失败！");
			return null;
		}
		Map<String, String> labels = new HashMap<String, String>();
		labels.put(OpenshiftUtil.UNIFIED_LABEL_KEY, application.getName());
		return labels;
	}

	private boolean saveApplicationLayoutInfo(ApplicationLayoutInfo applicationLayoutInfo, BsmResult bsmResult) {
		boolean result = false;
		try {
			result = applicationLayoutInfoDao.saveApplicationLayoutInfo(applicationLayoutInfo);
		} catch (Exception e) {
			bsmResult.setSuccess(false);
			logger.error("添加应用编排关联数据失败！", e);
			bsmResult.setMessage("添加应用编排关联数据失败！");
		}
		if (!result) {
			bsmResult.setSuccess(false);
			bsmResult.setMessage("添加应用编排关联数据失败！");
		}
		return result;
	}

	private boolean saveApplicationOpenshiftClusterInfo(ApplicationOpenshiftClusterInfo applicationOpenshiftClusterInfo,
			BsmResult bsmResult) {
		boolean result = false;
		try {
			result = applicationOpenshiftClusterInfoDao
					.saveApplicationOpenshiftClusterInfo(applicationOpenshiftClusterInfo);
		} catch (Exception e) {
			bsmResult.setSuccess(false);
			logger.error("添加应用集群关联数据失败！", e);
			bsmResult.setMessage("添加应用集群关联数据失败！");
		}
		if (!result) {
			bsmResult.setSuccess(false);
			bsmResult.setMessage("添加应用集群关联数据失败！");
		}
		return result;
	}

	private boolean saveDeployHistroy(DeployHistory deployHistory, BsmResult bsmResult) {
		boolean result = false;
		try {
			result = deployHistoryDao.baseSave(deployHistory);
		} catch (Exception e) {
			bsmResult.setSuccess(false);
			logger.error("添加部署历史数据失败！", e);
			bsmResult.setMessage("添加部署历史数据失败！");
		}
		if (!result) {
			bsmResult.setSuccess(false);
			bsmResult.setMessage("添加部署历史数据失败！");
		}
		return result;
	}

	/**
	 * 根据k8s资源集合添加rc或者deployment的部署历史记录和镜像数据
	 * 
	 * @author zjm
	 * @date 2017年4月6日
	 *
	 * @param sourceList
	 *            资源集合
	 * @param appId
	 *            应用id
	 * @param user
	 *            操作用户
	 * @return
	 */
	private void saveDeployHistroyAndSoftwareByResource(List<HasMetadata> sourceList, Long appId, BsmResult bsmResult,
			RequestUser user) {
		for (HasMetadata source : sourceList) {
			List<Container> containers = new ArrayList<>();
			String resourceType = "";
			if (source.getKind().equals("ReplicationController")) {
				ReplicationController rc = (ReplicationController) source;
				containers = rc.getSpec().getTemplate().getSpec().getContainers();
				resourceType = "ReplicationController";
				// TODO 暂不支持Deployment资源的管理，请不要删除
				// } else if (source.getKind().equals("Deployment")) {
				// Deployment deployment = (Deployment) source;
				// containers =
				// deployment.getSpec().getTemplate().getSpec().getContainers();
				// resourceType = "Deployment";
			} else {
				continue;
			}

			for (Container container : containers) {
				String image = container.getImage();
				String softwareNamespace = "";
				// 将镜像信息去除命名空间和ip地址等信息
				if (image.contains("/")) {
					if (image.substring(0, image.indexOf("/")).contains(":5000")) {
						softwareNamespace = (image.substring(image.indexOf("/") + 1));
						if (softwareNamespace.contains("/")) {
							softwareNamespace = softwareNamespace.substring(0, softwareNamespace.indexOf("/"));
						} else {
							softwareNamespace = "";
						}
						image = (image.substring(image.indexOf("/") + 1))
								.substring((image.substring(image.indexOf("/") + 1).indexOf("/") + 1));
					} else {
						softwareNamespace = image.substring(0, image.indexOf("/"));
						image = image.substring(image.indexOf("/") + 1);
					}
				}
				DeployHistory deployHistory = new DeployHistory();
				deployHistory.setStatus(DeployHistoryEnum.Status.FAIL.toString());
				if (bsmResult.isSuccess()) {
					// 判断应用镜像中间表是否包含这个应用和镜像，如果不包含添加应用镜像中间表数据
					String softwareName = image;
					String softwareVersion = "latest";
					if (image.contains(":")) {
						softwareName = image.substring(0, image.lastIndexOf(":"));
						softwareVersion = image.substring(image.lastIndexOf(":") + 1, image.length());
					}
					Software software = new Software();
					try {
						software = softwareDao.getSoftwareByNameWithVersionAndNamespace(softwareName, softwareVersion,
								softwareNamespace);
					} catch (Exception e) {
						bsmResult.setSuccess(false);
						bsmResult.setMessage("获取应用镜像[" + softwareNamespace + "/" + softwareName + ":" + softwareVersion
								+ "]失败！" + e);
						return;
					}
					if (null == software) {
						bsmResult.setSuccess(false);
						bsmResult.setMessage(
								"获取应用镜像[" + softwareNamespace + "/" + softwareName + ":" + softwareVersion + "]为空！");
						return;
					}
					ApplicationSoftwareInfo applicationSoftwareInfo = queryApplicationSoftwareByAppIdAndSoftwareId(
							appId, software.getId(), new BsmResult());
					if (null == applicationSoftwareInfo) {
						applicationSoftwareInfo = new ApplicationSoftwareInfo();
						applicationSoftwareInfo.setSoftwareId(software.getId());
						applicationSoftwareInfo.setApplicationId(appId);
						applicationSoftwareInfo.setUseCount(1);
						applicationSoftwareInfo.setSoftwareName(container.getImage());
						if (!saveApplicationSoftwareInfo(applicationSoftwareInfo, bsmResult)) {
							bsmResult.setSuccess(false);
							bsmResult.setMessage("添加应用镜像关联数据失败！");
							return;
						}
					} else {
						applicationSoftwareInfo.setUseCount(applicationSoftwareInfo.getUseCount() + 1);
						try {
							applicationSoftwareInfoDao.updateApplicationSoftwareInfo(applicationSoftwareInfo);
						} catch (Exception e) {
							bsmResult.setSuccess(false);
							bsmResult.setMessage("添加应用镜像关联数据失败！" + e);
							return;
						}
					}
					deployHistory.setStatus(DeployHistoryEnum.Status.SUCCESS.toString());
				}

				// 添加部署历史
				deployHistory.setResourceType(resourceType);
				deployHistory.setAppId(appId);
				deployHistory.setName(source.getMetadata().getName());
				deployHistory.setResourceName(source.getMetadata().getName());
				deployHistory.setCurrentVersion(image);
				deployHistory.setDeleted(false);
				deployHistory.setGmtCreate(new Date());
				deployHistory.setGmtModify(new Date());
				deployHistory.setCreaterId(user.getId());
				deployHistory.setMenderId(user.getId());
				deployHistory.setOwnerId(user.getId());
				deployHistory.setTenantId(user.getTenantId());
				saveDeployHistroy(deployHistory, bsmResult);
				if (!bsmResult.isSuccess()) {
					return;
				}
			}
		}
	}

	// TODO 暂不支持Deployment资源的管理，请不要删除
	// private BsmResult deleteDeployments(String proxyUrl, Map<String, String>
	// appLabels) {
	// BsmResult bsmResult = oseUtil.getDeployments(proxyUrl, appLabels);
	// @SuppressWarnings("all")
	// List<Deployment> deployments = (List<Deployment>) bsmResult.getData();
	// if (!ListTool.isEmpty(deployments)) {
	// if (!(bsmResult = oseUtil.deleteDeployments(proxyUrl,
	// deployments)).isSuccess()) {
	// bsmResult.setMessage("应用删除失败！" + bsmResult.getMessage());
	// }
	// }
	// return bsmResult;
	// }

	private BsmResult deleteServices(String proxyUrl, Map<String, String> appLabels) {
		BsmResult bsmResult = oseUtil.getServices(proxyUrl, appLabels);
		@SuppressWarnings("all")
		List<io.fabric8.kubernetes.api.model.Service> services = (List<io.fabric8.kubernetes.api.model.Service>) bsmResult
				.getData();
		if (!ListTool.isEmpty(services)) {
			if (!(bsmResult = oseUtil.deleteServices(proxyUrl, services)).isSuccess()) {
				bsmResult.setSuccess(false);
				bsmResult.setMessage("应用删除失败！" + bsmResult.getMessage());
			}
		}
		return bsmResult;
	}

	private BsmResult deletePods(String proxyUrl, Map<String, String> appLabels) {
		BsmResult bsmResult;
		OpenshiftUtil.TwoTuple<List<Pod>, String> twoTuple = oseUtil.getPods(proxyUrl, appLabels);
		List<Pod> pods = twoTuple.first;
		if (!ListTool.isEmpty(pods)) {
			if (!(bsmResult = oseUtil.deletePods(proxyUrl, pods)).isSuccess()) {
				bsmResult.setMessage("应用删除失败！" + bsmResult.getMessage());
			}
		} else {
			bsmResult = new BsmResult(true, twoTuple.second);
		}
		return bsmResult;
	}

	private BsmResult deleteHpas(String proxyUrl, Map<String, String> appLabels) {
		BsmResult bsmResult = new BsmResult();
		OpenshiftUtil.TwoTuple<List<HorizontalPodAutoscaler>, String> twoTuple = oseUtil.getHPAs(proxyUrl, appLabels);
		List<HorizontalPodAutoscaler> hpas = twoTuple.first;
		if (!ListTool.isEmpty(hpas)) {
			if (!(bsmResult = oseUtil.deleteHPAs(proxyUrl, hpas)).isSuccess()) {
				bsmResult.setMessage("hpa删除失败！" + bsmResult.getMessage());
			}
		} else {
			bsmResult = new BsmResult(true, twoTuple.second);
		}
		return bsmResult;
	}

	@Override
	public BsmResult getPodsByAppId(Long appId) {
		BsmResult bsmResult = new BsmResult();
		Map<String, String> appLabels = getAppLabels(appId, bsmResult);
		if (null == appLabels) {
			return bsmResult;
		}
		bsmResult = getClusterProXyUrl(appId);
		if (bsmResult.isSuccess() && null != bsmResult.getData()) {
			bsmResult.setData(oseUtil.getPods(bsmResult.getData().toString(), appLabels).first);
		}
		return bsmResult;
	}

	@Override
	public BsmResult getReplicas(Long appId, String namespace, String resourceName, String resourceType,
			RequestUser user) {
		BsmResult bsmResult = getClusterProXyUrl(appId);
		if (!bsmResult.isSuccess() || null == bsmResult.getData()) {
			saveLog("application/operation", "获取实例数", bsmResult.getMessage(), user.getId());
			return bsmResult;
		}
		String proxyUrl = bsmResult.getData().toString();
		if (resourceType.equals("ReplicationController")) {
			bsmResult = oseUtil.getRc(proxyUrl, namespace, resourceName);
			if (!bsmResult.isSuccess() || null == bsmResult.getData()) {
				saveLog("application/operation", "获取实例数", bsmResult.getMessage(), user.getId());
				return bsmResult;
			}
			ReplicationController rc = (ReplicationController) bsmResult.getData();
			bsmResult = oseUtil.getRcsReplicas(proxyUrl, rc.getMetadata().getLabels());
			saveLog("application/operation", "获取实例数", bsmResult.getMessage(), user.getId());
			return bsmResult;
		}
		bsmResult.setSuccess(false);
		bsmResult.setMessage("不支持查看该资源的实例数！ ");
		saveLog("application/operation", "获取实例数", bsmResult.getMessage(), user.getId());
		return bsmResult;
	}

	@Override
	public BsmResult getClusterProXyUrl(Long appId) {
		BsmResult bsmResult = new BsmResult();
		ApplicationOpenshiftClusterInfo applicationOpenshiftClusterInfo = null;
		try {
			applicationOpenshiftClusterInfo = applicationOpenshiftClusterInfoDao
					.getApplicationOpenshiftClusterInfo(appId);
		} catch (Exception e) {
			logger.error("获取应用集群中间表失败！", e);
			bsmResult.setMessage("获取应用集群中间表失败！");
		}
		if (null != applicationOpenshiftClusterInfo) {
			OpenshiftCluster openshiftCluster = null;
			try {
				openshiftCluster = openshiftClusterDao.query(applicationOpenshiftClusterInfo.getOpenshiftClusterId());
			} catch (Exception e) {
				logger.error("获取应用集群失败！", e);
				bsmResult.setMessage("获取应用集群失败！");
			}
			if (null != openshiftCluster) {
				bsmResult.setSuccess(true);
				bsmResult.setData(openshiftCluster.getProxyUrl());
				bsmResult.setMessage("获取集群url成功！");
			} else {
				bsmResult.setMessage("获取应用集群失败！");
			}
		} else {
			bsmResult.setMessage("获取应用集群中间表失败！");
		}
		return bsmResult;
	}

	@Transactional
	public boolean removeAppInfos(Long id, Long userId) {
		try {
			return applicationDao.deleteById(id, userId) && applicationDao.deleteAppLayoutInfo(id)
					&& applicationDao.deleteAppSoftwareInfo(id) && applicationDao.delateAppClusterInfo(id);
		} catch (Exception e) {
			logger.error("Remove application infos exception:", e);
			return false;
		}
	}

	private boolean modifyApplication(Application application, BsmResult bsmResult) {
		boolean result = false;
		try {
			result = applicationDao.update(application);
		} catch (Exception e) {
			logger.error("更新应用数据失败！", e);
			bsmResult.setSuccess(false);
			bsmResult.setMessage("更新应用数据失败！");
		}
		if (!result) {
			bsmResult.setSuccess(false);
			bsmResult.setMessage("更新应用数据失败！");
		}
		return result;
	}

	private boolean saveApplicationSoftwareInfo(ApplicationSoftwareInfo applicationSoftwareInfo, BsmResult bsmResult) {
		boolean result = false;
		try {
			result = applicationSoftwareInfoDao.saveApplicationSoftwareInfo(applicationSoftwareInfo);
		} catch (Exception e) {
			logger.error("添加应用镜像关联数据失败！", e);
			bsmResult.setSuccess(false);
			bsmResult.setMessage("添加应用镜像关联数据失败！");
		}
		if (!result) {
			bsmResult.setSuccess(false);
			bsmResult.setMessage("添加应用镜像关联数据失败！");
		}
		return result;
	}

	private boolean deleteApplicationSoftwareInfo(ApplicationSoftwareInfo applicationSoftwareInfo,
			BsmResult bsmResult) {
		boolean result = false;
		try {
			result = applicationSoftwareInfoDao.deleteApplicationSoftwareInfo(applicationSoftwareInfo);
		} catch (Exception e) {
			logger.error("删除应用镜像关联数据失败！", e);
			bsmResult.setSuccess(false);
			bsmResult.setMessage("删除应用镜像关联数据失败！");
		}
		if (!result) {
			bsmResult.setSuccess(false);
			bsmResult.setMessage("删除应用镜像关联数据失败！");
		}
		return result;
	}

	/**
	 * 添加应用中间表
	 * 
	 * @author zjm
	 * @date 2017年4月25日
	 *
	 * @param application
	 * @param clusterId
	 * @return
	 */
	@Transactional
	public BsmResult insertApplicationMiddleTable(Application application, Long layoutId, Long clusterId) {
		BsmResult bsmResult = new BsmResult();
		ApplicationLayoutInfo applicationLayoutInfo;
		try {
			applicationLayoutInfo = applicationLayoutInfoDao.getByAppIdAndLayoutId(application.getId(), layoutId);
		} catch (Exception e) {
			logger.error("获取应用编排中间表失败！", e);
			bsmResult.setMessage("获取应用编排中间表失败！");
			return bsmResult;
		}
		if (null == applicationLayoutInfo) {
			applicationLayoutInfo = new ApplicationLayoutInfo();
			applicationLayoutInfo.setApplicationId(application.getId());
			applicationLayoutInfo.setLayoutId(layoutId);
			if (!saveApplicationLayoutInfo(applicationLayoutInfo, bsmResult)) {
				logger.warn(bsmResult.getMessage());
				return bsmResult;
			}
		}
		// 添加应用集群中间表数据
		// 判断应用是否在该集群中部署过，防止应用集群信息重复
		ApplicationOpenshiftClusterInfo applicationOpenshiftClusterInfo = queryApplicationOpenshiftClusterInfo(
				application.getId(), bsmResult);
		if (null == applicationOpenshiftClusterInfo) {
			applicationOpenshiftClusterInfo = new ApplicationOpenshiftClusterInfo();
			applicationOpenshiftClusterInfo.setApplicationId(application.getId());
			applicationOpenshiftClusterInfo.setOpenshiftClusterId(clusterId);
			if (!saveApplicationOpenshiftClusterInfo(applicationOpenshiftClusterInfo, bsmResult)) {
				logger.warn(bsmResult.getMessage());
				return bsmResult;
			}
		}
		application.setStatus(ApplicationEnum.Status.DEPLOY.toString());
		if (!modifyApplication(application, bsmResult)) {
			logger.warn(bsmResult.getMessage());
		}
		bsmResult.setSuccess(true);
		return bsmResult;
	}

	@Override
	public BsmResult rebuildServices(String namespace, String resourceName, Long appId, Long softwareId,
			RequestUser user) {
		String operation = "重建服务";
		BsmResult bsmResult = new BsmResult();
		bsmResult = getClusterProXyUrl(appId);
		if (!bsmResult.isSuccess() || null == bsmResult.getData()) {
			saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
			return bsmResult;
		}
		String proxyUrl = bsmResult.getData().toString();
		try {
			// 启动一个线程从MQ上获取数据
			new Thread(new Runnable() {
				@Override
				public void run() {
					BsmResult bsmResult = new BsmResult();
					Map<String, String> appLabels = getAppLabels(appId, bsmResult);
					bsmResult = oseUtil.getServices(proxyUrl, appLabels);
					if (!bsmResult.isSuccess()) {
						bsmResult.setMessage(operation + "失败！" + bsmResult.getMessage());
						logger.error(bsmResult.getMessage());
						saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
						resourceEventPublisher.send(new OperateResult(bsmResult.isSuccess(), bsmResult.getMessage(),
								"application/rebuildServices", bsmResult.getData(), user.getId()));
						return;
					}
					@SuppressWarnings("unchecked")
					List<io.fabric8.kubernetes.api.model.Service> services = (List<io.fabric8.kubernetes.api.model.Service>) bsmResult
							.getData();
					// 复制service
					for (io.fabric8.kubernetes.api.model.Service service : services) {
						io.fabric8.kubernetes.api.model.Service newService = new ServiceBuilder(service).build();
						newService.getMetadata().setResourceVersion("");
						bsmResult = oseUtil.deleteService(proxyUrl, service);
						try {
							Thread.sleep(rollingSleep * 1000);
						} catch (InterruptedException e) {
							bsmResult.setMessage(operation + "失败！睡眠异常！");
							logger.error(bsmResult.getMessage());
							saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
							resourceEventPublisher.send(new OperateResult(bsmResult.isSuccess(), bsmResult.getMessage(),
									"application/rebuildServices", bsmResult.getData(), user.getId()));
							return;
						}
						if (bsmResult.isSuccess()) {
							if (bsmResult.isSuccess()) {
								bsmResult = oseUtil.createService(proxyUrl, newService);
							} else {
								bsmResult.setMessage(operation + "失败！" + bsmResult.getMessage());
								logger.error(bsmResult.getMessage());
								saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
								resourceEventPublisher
										.send(new OperateResult(bsmResult.isSuccess(), bsmResult.getMessage(),
												"application/rebuildServices", bsmResult.getData(), user.getId()));
								return;
							}
						} else {
							bsmResult.setMessage(operation + "失败！" + bsmResult.getMessage());
							logger.error(bsmResult.getMessage());
							saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
							resourceEventPublisher.send(new OperateResult(bsmResult.isSuccess(), bsmResult.getMessage(),
									"application/rebuildServices", bsmResult.getData(), user.getId()));
							return;
						}
					}
					if (bsmResult.isSuccess()) {
						bsmResult.setMessage(operation + "成功！");
						logger.info(bsmResult.getMessage());
					} else {
						bsmResult.setMessage(operation + "失败！" + bsmResult.getMessage());
						logger.error(bsmResult.getMessage());
					}

					saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
					resourceEventPublisher.send(new OperateResult(bsmResult.isSuccess(), bsmResult.getMessage(),
							"application/rebuildServices", bsmResult.getData(), user.getId()));
					return;
				}
			}).start();
			return new BsmResult(true, operation + "任务已经下发，正在执行……");
		} catch (Exception e) {
			logger.error("application rebuildServices failure:", e);
			bsmResult.setSuccess(false);
			bsmResult.setMessage(operation + "失败!");
			saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
			return bsmResult;
		}
	}

	@Override
	public BsmResult upgrade(String namespace, String resourceName, Long appId, Long softwareId, RequestUser user) {
		String operation = "全量升级";
		BsmResult bsmResult = new BsmResult();

		if (null == (bsmResult = getClusterProXyUrl(appId)).getData()) {
			bsmResult.setSuccess(false);
			bsmResult.setMessage("全量升级失败！");
			saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
			return bsmResult;
		}
		String proxyUrl = bsmResult.getData().toString();
		Application application;
		if (null == (application = queryApplication(appId, bsmResult))) {
			saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
			return bsmResult;
		}

		try {
			// 启动一个线程从MQ上获取数据
			new Thread(new Runnable() {
				@Override
				public void run() {
					BsmResult bsmResult = new BsmResult();
					// 当前rc
					ReplicationController currentRc = null;
					Software software = new Software();
					String imageName = "";
					if (null == (software = querySoftware(softwareId, new BsmResult()))) {
						bsmResult.setSuccess(false);
						bsmResult.setMessage("全量升级失败！");
						saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
						resourceEventPublisher.send(new OperateResult(false, bsmResult.getMessage(),
								"application/layoutDeploy", user.getId()));
						return;
					}
					RegistryImageInfo registryImageInfo = new RegistryImageInfo();
					if (null == (registryImageInfo = queryRegistryImageInfo(softwareId, bsmResult))) {
						bsmResult.setMessage("全量升级失败！");
						saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
						resourceEventPublisher.send(new OperateResult(false, bsmResult.getMessage(),
								"application/layoutDeploy", user.getId()));
						return;
					}
					if (StringUtils.isNotBlank(registryImageInfo.getNamespace())) {
						imageName = registryImageInfo.getNamespace() + "/" + software.getName() + ":"
								+ software.getVersion();
					} else {
						imageName = software.getName() + ":" + software.getVersion();
					}
					if (StringUtils.isBlank(imageName)) {
						bsmResult.setSuccess(false);
						bsmResult.setMessage("全量升级失败！");
						saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
						resourceEventPublisher.send(new OperateResult(false, bsmResult.getMessage(),
								"application/layoutDeploy", user.getId()));
						return;
					}
					if (null == (bsmResult = oseUtil.getRc(proxyUrl, namespace, resourceName)).getData()) {
						bsmResult.setSuccess(false);
						bsmResult.setMessage("全量升级失败！");
						saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
						resourceEventPublisher.send(new OperateResult(false, bsmResult.getMessage(),
								"application/layoutDeploy", user.getId()));
						return;
					}
					currentRc = (ReplicationController) bsmResult.getData();
					if (currentRc == null) {
						bsmResult.setSuccess(false);
						bsmResult.setMessage("全量升级失败！");
						saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
						resourceEventPublisher.send(new OperateResult(false, bsmResult.getMessage(),
								"application/layoutDeploy", user.getId()));
						return;
					}
					// 获取当前rc，并复制新的rc修改名称、镜像和标签，然后部署修改后的rc
					String newRcName = currentRc.getMetadata().getName();
					if (newRcName.indexOf("-") > 0) {
						newRcName = newRcName.substring(0, newRcName.lastIndexOf("-")) + "-"
								+ RandomStringUtils.randomAlphanumeric(4).toLowerCase();
					} else {
						newRcName += "-" + RandomStringUtils.randomAlphanumeric(4).toLowerCase();
					}
					ReplicationController newRc = oseUtil.cloneRc(proxyUrl, currentRc, imageName, newRcName);
					newRc.getSpec().getTemplate().getSpec().getContainers().get(0).setImagePullPolicy("IfNotPresent");
					if (!(bsmResult = oseUtil.createReplicationController(proxyUrl, newRc)).isSuccess()) {
						bsmResult.setMessage("全量升级失败！");
						saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
						resourceEventPublisher.send(new OperateResult(false, bsmResult.getMessage(),
								"application/layoutDeploy", user.getId()));
						return;
					}
					if (!(bsmResult = oseUtil.getRc(proxyUrl, namespace, newRcName)).isSuccess()) {
						bsmResult.setMessage("全量升级失败！");
						saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
						resourceEventPublisher.send(new OperateResult(false, bsmResult.getMessage(),
								"application/layoutDeploy", user.getId()));
						return;
					}
					// 判断新的rc的状态，如果不健康则删除新的rc，如果健康则删除老的rc
					List<ReplicationController> replicationControllers = new ArrayList<ReplicationController>();
					if (!(bsmResult = oseUtil.checkRcReady(proxyUrl, newRc)).isSuccess()) {
						@SuppressWarnings("unchecked")
						List<Pod> pods = (List<Pod>) bsmResult.getData();
						String logResult = "";
						for (Pod pod : pods) {
							bsmResult = statisticAppServiceImpl.event(pod.getMetadata().getNamespace(), pod.getKind(),
									pod.getMetadata().getName(), application.getClusterId());
							if (bsmResult.isSuccess() && null != bsmResult.getData()) {
								logResult += bsmResult.getData().toString();
							}
						}
						replicationControllers.add(newRc);
						if (!(bsmResult = oseUtil.deleteReplicationControllers(proxyUrl, replicationControllers))
								.isSuccess()) {
							bsmResult.setMessage("全量升级失败！退回旧版本失败，请手动删除！");
						} else {
							bsmResult.setMessage("全量升级失败！已退回旧版本");
						}
						bsmResult.setSuccess(false);
						logger.error("全量升级失败：\n" + logResult);
						saveLog("application/operation", operation, logResult, user.getId());
						resourceEventPublisher.send(new OperateResult(bsmResult.isSuccess(), bsmResult.getMessage(),
								"application/layoutDeploy", user.getId()));
						return;
					} else {
						replicationControllers.add(currentRc);
						if (!(bsmResult = oseUtil.deleteReplicationControllers(proxyUrl, replicationControllers))
								.isSuccess()) {
							bsmResult.setMessage("全量升级成功！删除旧版本失败，请手动删除！");
						} else {
							bsmResult.setMessage("全量升级成功！");
						}

						// 添加修改应用相关中间表数据
						List<HasMetadata> hasMetadatas = new ArrayList<HasMetadata>();
						hasMetadatas.add(newRc);
						saveDeployHistroyAndSoftwareByResource(hasMetadatas, appId, bsmResult, user);
						deleteSurplusImage(appId);
						saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
						resourceEventPublisher.send(new OperateResult(bsmResult.isSuccess(), bsmResult.getMessage(),
								"application/layoutDeploy", user.getId()));
						return;
					}
				}
			}).start();
			return new BsmResult(true, operation + "任务已经下发，正在执行……");
		} catch (Exception e) {
			logger.error("application rebuildServices failure:", e);
			bsmResult.setSuccess(false);
			bsmResult.setMessage(operation + "失败!");
			saveLog("application/operation", operation, bsmResult.getMessage(), user.getId());
			return bsmResult;
		}

	}

	/**
	 * 江苏银行特殊需求，判断当前用户是否是系统管理员或者拥有查看“容器平台的权限”
	 * 
	 * @param user
	 * @return
	 */
	private boolean checkAuthority(RequestUser user) {
		try {
			List<Role> roles = roleDao.listByUid(user.getId());
			for (Role role : roles) {
				List<Authority> authorities = authDao.listByRid(role.getId());
				for (Authority authority : authorities) {
					// TODO 和威哥少龙沟通后暂时写死判断“容器平台”权限的id
					// 具有“容器平台”的人具有“特殊”权限
					if (role.getId().equals(new Long(1)) && authority.getId().equals(new Long(132))) {
						return true;
					}
				}
			}
		} catch (Exception e) {
			return false;
		}
		return false;
	}

	/**
	 * 删除当前应用的所有未接管的镜像
	 * 
	 * @param appId
	 */
	@SuppressWarnings("unchecked")
	private void deleteSurplusImage(Long appId) {
		BsmResult bsmResult = new BsmResult();
		// 应用下的所有依赖镜像
		Map<Long, String> appImages = new HashMap<Long, String>();
		List<ApplicationSoftwareInfo> applicationSoftwareInfos = queryApplicationSoftwareByAppId(appId);
		if (ListTool.isEmpty(applicationSoftwareInfos)) {
			return;
		}
		for (ApplicationSoftwareInfo applicationSoftwareInfo : applicationSoftwareInfos) {
			appImages.put(applicationSoftwareInfo.getId(), applicationSoftwareInfo.getSoftwareName());
		}
		// 应用下rc的镜像
		List<String> rcImages = new ArrayList<String>();
		bsmResult = getClusterProXyUrl(appId);
		if (!bsmResult.isSuccess() || null == bsmResult.getData()) {
			logger.error(bsmResult.getMessage());
			return;
		}
		Map<String, String> labels = getAppLabels(appId, new BsmResult());
		bsmResult = oseUtil.getRcs(bsmResult.getData().toString(), labels);
		if (!bsmResult.isSuccess() || null == bsmResult.getData()) {
			logger.error(bsmResult.getMessage());
			return;
		}
		for (ReplicationController rc : (List<ReplicationController>) bsmResult.getData()) {
			List<Container> containers = rc.getSpec().getTemplate().getSpec().getContainers();
			for (Container container : containers) {
				rcImages.add(container.getImage());
			}
		}
		for (Long applicationSoftwarId : appImages.keySet()) {
			if (!rcImages.contains(appImages.get(applicationSoftwarId))) {
				ApplicationSoftwareInfo applicationSoftwareInfo;
				try {
					applicationSoftwareInfo = applicationSoftwareInfoDao.get(applicationSoftwarId);
				} catch (Exception e) {
					continue;
				}
				deleteApplicationSoftwareInfo(applicationSoftwareInfo, bsmResult);
			}
		}

	}

	private List<ApplicationSoftwareInfo> queryApplicationSoftwareByAppId(Long appId) {
		try {
			return applicationSoftwareInfoDao.getByAppId(appId);
		} catch (Exception e) {
			logger.error("获取应用镜像信息失败：\n", e);
		}
		return null;
	}

}
